#coding=utf8
import sys
import jieba.analyse
import jieba.posseg as pseg
from config.config import Config
conf=Config()
home_dir =  conf.get_home_dir()
home_data_dir = conf.get_home_data_dir()
dictPath = '/home/yzfu/nlp/kg_abc/fyz_kg_nlp/abc_project_data/new_dict.txt'
stopword = '/home/yzfu/nlp/kg_abc/fyz_kg_nlp/abc_project_data/stopword.dict'
#mark_path = home_data_dir+'mark.txt'
#print "mark_path :{}".format(mark_path)
jieba.load_userdict(dictPath)
jieba.analyse.set_stop_words(stopword)  # file_name为自定义语料库的路径


reload(sys)
sys.setdefaultencoding( "utf-8" )

#mark_path = home_dir + 'result/mark.txt'



#获取所有的已标注数据
def get_mark_info(mark_path):
    file = open(mark_path, 'r')
    lines = file.readlines()
    markdatas = []
    for line in lines:
        line = line.split(' ')
        markdatas.append(line)
    return markdatas


"""
def hash_fyz_pos_word(path,tag_output,mark_dict):
    #读取文件
    file = open(path,'rU').read()
    file = str(file).replace('\\n', "\n").replace('\\','')
    #词性标注
    words = pseg.cut(str(file))
    tag_list = []
    #统计标注还原的数量
    for word,flag in words:
        pos=''
        if word.strip()=="" or word==" " : continue
        if  word.strip()=='\n':
            tag_list.append(word)
        else:
            try:
                if hash(word.encode('utf-8')) in mark_dict.keys():
                    pos=flag
                    flag=mark_dict[hash(word.encode('utf-8'))]
            except KeyError :
                flag=pos
            tag_list.append(word + '/' + flag+'\n')
    seg_list2 = " ".join(tag_list)
    #标注结果导出
    fout = open(tag_output, 'w')
    fout.write(seg_list2)
    fout.close()
"""
def hash_fyz_pos_word(path,tag_output,mark_dict):
    #读取文件
    file_data= open(path,'r').read()
    tag_list = []
    ner_token = []
    fout = open(tag_output, 'w')
    word_list = jieba.cut(file_data, cut_all=False)
    fout.write(" ".join(word_list))
    """
    for words in word_list:
        if words.strip()=="" or words==" " : continue
        if  words.strip()=='\n':
            tag_list.append(words)
        else:
            try:
                if hash(words.encode('utf-8')) in mark_dict.keys():
                    flag=mark_dict[hash(words.encode('utf-8'))]
            except KeyError :
                flag='O'
        BI_list = get2Tag(words)
        for char, BI in zip(list(words), BI_list):
            if flag != 'O':
                ner_token.append(char+" "+ BI + '-' + flag+'\n')
            else:
                ner_token.append(char + " " + flag + '\n')
    ner_token.append('\n')
    #标注结果导出
    fout = open(tag_output, 'w')
    fout.writelines(ner_token)
    fout.close()
    """

def fyz_pos_tag_word(line,mark_dict):
    words = pseg.cut(line)
    tag_list = []
    #统计标注还原的数量
    cal_num = []
    word_list = []
    result=[]
    for word,flag in words:
        pos = ''
        if word.strip()=="" or word==" " : continue
        if  word.strip()=='\n':
            tag_list.append(word)
        else:
            try:
                if hash(word.encode('utf-8')) in mark_dict.keys():
                    pos=flag
                    flag=mark_dict[hash(word.encode('utf-8'))]
            except KeyError :
                flag=pos
            tag_list.append(word + '/' + flag+'\n')
        BI_list = get2Tag(word)
        for char, BI in zip(list(word), BI_list):
            result.append([char, BI + '-' + flag])

        word_list.append(word)
    str = "".join(word_list)
    #seg_list2 = " ".join(tag_list)
    #print '标注还原的数量:',len(cal_num)
    #标注结果导出
    return tag_list,str,result


#保存生成的原始数据 train&test
def saveDataFile(trainobj,testobj,isTest,word,handle,tag):
    if isTest:
        saveTrainFile(testobj,word,handle,tag)
    else:
        saveTrainFile(trainobj,word,handle,tag)

#保存数据
def saveTrainFile(fiobj,word,handle,tag):
    if len(word) > 0 and  word != "。" and word != "，":
         fiobj.write(word + '\t' + handle  + '\t' +tag +'\n')
        # fiobj.write(word + '\t' + handle +  '\n')
    else:
        fiobj.write('\n')

def splitWord(words):
    uni = words.decode('utf-8')
    li = list()
    for u in uni:
        li.append(u.encode('utf-8'))
    return li

    # 4 tag

# B/I
def get2Tag(li):
    length = len(li)
    #print length
    if length == 1:
        return ['B']
    elif length == 2:
        return ['B', 'I']
    elif length > 2:
        li = list()
        li.append('B')
        for i in range(0, length - 2):
            li.append('I')
        li.append('I')
        return li

# S/B/E/M
def get4Tag(li):
    length = len(li)
    #print length
    if length == 1:
        return ['S']
    elif length == 2:
        return ['B', 'E']
    elif length > 2:
        li = list()
        li.append('B')
        for i in range(0, length - 2):
            li.append('M')
        li.append('E')
        return li

# 6 tag
# S/B/E/M/M1/M2
def get6Tag(li):
    length = len(li)
    #print length
    if length == 1:
        return ['S']
    elif length == 2:
        return ['B', 'E']
    elif length == 3:
        return ['B', 'M', 'E']
    elif length == 4:
        return ['B', 'M1', 'M', 'E']
    elif length == 5:
        return ['B', 'M1', 'M2', 'M', 'E']
    elif length > 5:
        li = list()
        li.append('B')
        li.append('M1')
        li.append('M2')
        for i in range(0, length - 4):
            li.append('M')
        li.append('E')
    return li

def saveLstmDataFile(trainobj, testobj, isTest, word, handle, tag):
    if isTest:
        saveLstmTrainFile(testobj, word, handle, tag)
    else:
        saveLstmTrainFile(trainobj, word, handle, tag)



#保存结果
def saveLstmTrainFile(fiobj, word, handle, tag):
    #print 'wordli:', word,' tag:',tag
    if len(word.strip()) > 0:
        wordli = splitWord(word)
        #print 'wordli:',wordli
    if tag =='2' :
        tagli = get2Tag(wordli)
    if tag == '4':
        tagli = get4Tag(wordli)
    if tag == '6':
        tagli = get6Tag(wordli)
    #print 'tagli:',tagli
    for i in range(0, len(wordli)):
        w = wordli[i].strip()
        h = handle
        t = tagli[i]
        #print 'w,t:',w,t
        # fiobj.write(w + '/t' + h + '/t' + t + '/n')
        content = str(w) + " " + str(changeTag(h,t)) + "\n"
        fiobj.write(content)
    else:
     # print 'New line'
        fiobj.write("\n" )

#替换标签
#ns 地点 LOC ,nr 人名 PER, nt 公司组织 ORG, v 关系 REL,attr 属性 VAL
def changeTag(handle,t):
    if handle=="nr":
        return t+'-person'
    elif handle=="nt":
        return t+'-agency'
    elif handle=="v":
        return 'O'
    elif handle=="ns":
        return t+'-location'
    elif handle=="t":
        return 'O'
    elif handle=="attr" :
        return 'O'
    elif handle=="person":
        return t+'-person'
    elif handle=="agency":
        return t+'-agency'
    elif handle=="product":
        return t+'-product'
    elif handle=="business":
        return t+'-business'
    elif handle=="industry":
        return t+'-industry'
    elif handle=="file":
        return 'O'
    else:
        return "O"

def fyz_changeTag(handle):
    if handle=="nr":
        return 'person'
    elif handle=="nt":
        return 'agency'
    elif handle=="ns":
        return 'location'
    elif handle=="person":
        return 'person'
    elif handle=="agency":
        return 'agency'
    elif handle=="product":
        return 'product'
    elif handle=="business":
        return 'business'
    elif handle=="industry":
        return 'industry'
    else:
        return "O"

"""
# B,M,M1,M2,M3,E,S
def lstm_convertTag(fiobj,trainobj,testobj,tag='6'):
    fiobj = open(fiobj, 'r')
    trainobj = open(trainobj, 'wb+')
    testobj = open(testobj, 'wb+')
    i = 0
    for a in fiobj.readlines():
        i += 1
        if a=="\n": continue
        a = a.strip('/r/n/t// ')
        words = a.split(' ')
        test = False
        # if i % 10 == 0:
        #     test = True
        for word in words:
            word = word.strip('\n ')
            if len(word) > 0:
                # i1 = word.find('[')
                # if i1 >= 0:
                #     word = word[i1 + 1:]
                # i2 = word.find(']')
                # if i2 > 0:
                #     word = word[:i2]
                #print word
                word_hand = word.split('/')
                if len(word_hand)>2 or len(word_hand)<2 :continue
                w, h = word_hand

                # print w,h
                if h == 'nr':  # ren min
                    # print 'NR',w
                    if w.find('·') >= 0:
                        tmpArr = w.split('·')
                        for tmp in tmpArr:
                            saveLstmDataFile(trainobj, testobj, test, tmp, h, tag)
                        continue
                    else:
                        saveLstmDataFile(trainobj, testobj, test, w, h, tag)
                if h != 'm':
                    saveLstmDataFile(trainobj, testobj, test, w, h, tag)

                if h == 'w':
                    saveLstmDataFile(trainobj, testobj, test, "", "", tag)  # split

    trainobj.flush()
    testobj.flush()

"""


#填充标注，非的不添加
#ns 地点 LOC ,nr 人名 PER, nt 公司组织 ORG, v 关系 REL,attr 属性 VAL
def fill_local_tag(words, tags):
    pos = 0
    while True:
        #print "pos:", pos, " len:", len(words)," location:",words
        if pos == len(words):
            #print "添加tag执行结束"
            #print tags
            break
        word = words[pos]
        left = word.find("[")
        if left == -1 :
            #print "单个词", word
            if word=="//x": break
            w,h = word.split("/")
            #print w,h
            prefix = ""
            if h == "ns": #单个词是地点
                prefix = "location"
            if h=="nr":
                prefix="person";
            if h=="nt":
                prefix="agency"
                tags[pos] = prefix + "_S"
            #print "本轮tag",tags[pos]
            pos += 1
        elif left >= 0:
            #print "发现词组" ,word
            search_pos = pos
            for word in words[pos+1:]:
                #print word
                search_pos += 1
                if word.find("[") >=0:
                    #print "括号配对异常"
                    sys.exit(255)
                if word.find("]") >=0:
                    break
            if words[search_pos].find("]")  == -1:
                #print "括号配对异常，搜索到句尾没有找都另一半括号"
                sys.exit(255)
            else:
                #找到另一半，判断原始标注是不是ns，如果是就进行tag标注
                #print "match到一个组", words[pos:search_pos+1]
                h = words[search_pos].split("]")[-1] #最后一个词性
                if h == "ns" or h == "nr" or h == "nt" or h == "atr" or h == "v":
                    tags[pos] = prefix+"_B" #添加首个词
                    for p in range(pos + 1,search_pos + 1):
                      tags[p] = prefix+"_I" #中间词
                    tags[search_pos] = prefix+"_E" #找到最后一个词
                else:
                    p = pos
                    for word in words[pos:search_pos+1]:
                        #print "hhhhhhh", word
                        if len(word.strip("[").split("]")[0].split("/")) >1:
                            w,h = word.strip("[").split("]")[0].split("/")
                            if h == "ns" or h=="nr" or h=="nt" or h=="atr" or h=="v":
                                tags[p] = prefix+"_S"
                            p += 1
            #移动pos
            #print "本轮添加的tag", tags[pos:search_pos+1]
            pos = search_pos + 1

def convertTag(fiobj,trainobj,testobj,size=5,isTest=True):
    fiobj = open(fiobj, 'r')
    trainobj = open(trainobj, 'wb+')
    testobj = open(testobj, 'wb+')

    arr = fiobj.readlines()
    i = 0
    for a in arr:
        i += 1
        a = a.strip('\r\n\t ')
        if a=="":continue
        words = a.split(" ")
        test = False
        if i % 5 == 0 & isTest==True:
            #print test
            test = True
        words = words[1:]
        if len(words) == 0: continue

        tags = ["O"] * len(words)
        fill_local_tag(words, tags)

        pos = -1
        for word in words:
            pos += 1
            if word == "//x": continue
            #print "---->", word
            word = word.strip('\t ')
            if len(word) == 0:
                #print "Warning 发现空词"
                continue

            l1 = word.find('[')
            if l1 >=0:
                word = word[l1+1:]

            l2 = word.find(']')
            if l2 >= 0:
                word = word[:l2]

            if len(word.split('/'))<=1: continue
            w,h = word.split('/')

            saveDataFile(trainobj,testobj,test,w,h,tags[pos])
        saveDataFile(trainobj, testobj, test,"","","")

    trainobj.flush()
    testobj.flush()

#生产数据
def productTag(self):
    path = home_dir+'train_result_tag.txt'
    trainpath = home_dir+'tag_train.data'
    testpath=home_dir+'tag_test.data'
    convertTag(path, trainpath, testpath, size=5,isTest=False)

#训练及测试数据
def trainTag(self):
    path = home_dir+'train_result_tag.txt'
    trainpath = home_dir+'tag_train.data'
    testpath = home_dir+'tag_test.data'
    convertTag(path,trainpath,testpath,size=5)

